package il.ac.tau.dbcourse.freedb;

import il.ac.tau.dbcourse.db.ConnectionFactory;
import il.ac.tau.dbcourse.freedb.FreeDbFileReader.KeyValue;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tools.bzip2.CBZip2InputStream;

import com.ice.tar.TarEntry;
import com.ice.tar.TarInputStream;

// Implements a FreeDB Parser using the spec found at: http://ftp.freedb.org/pub/freedb/misc/freedb_database_format_specs.zip
public class FreeDbParser {	
	public static class Track {						
		String artist;
		String title;		
		int length;				
		
		public Track(String artist, String title, int length) {
			super();
			this.artist = artist;
			this.title = title;
			this.length = length;
		}
		
		public String getArtist() {
			return artist;
		}
		
		public String getTitle() {
			return title;
		}
		
		public int getLength() {
			return length;
		}
		
		@Override
		public String toString() {										
			if (artist != null)			
				return artist + " - " + title + " - " + length;
		
			return title + " - " + length;		
		}
	}
	
	private static final String TRACK_OFFSETS_STR = "# Track frame offsets:";
	private static final String DISC_LENGTH_STR = "# Disc length:";	 
	private static final String REVISION_STR = "# Revision:";
	private static final String TRACK_TITLE = "TTITLE";		
	private static final String[] ignoreList = new String[] { "README", "COPYING" };
	
	static PrintStream printStream; 
	
	private FreeDbDataLoader freeDbDataLoader;	
	private int albumsCounter = 0;
	private String errorPath;
	private final List<String> lines = new ArrayList<String>();
	private boolean writeErrorFiles;
	private int fileCounter = 0;
	private boolean recursive;
	private static int maxAlbums = 1000000; //Integer.MAX_VALUE;
	
	public FreeDbParser(FreeDbDataLoader freeDbDataLoader, String errorPath, boolean writeErrorFiles, boolean recursive) {
		this.freeDbDataLoader = freeDbDataLoader;
		this.errorPath = errorPath;
		this.writeErrorFiles = writeErrorFiles;
		this.recursive = recursive;
	}
	
	public void resetAlbumsCounter() {
		albumsCounter = 0;
	}
	
	protected boolean processAlbum(File file, InputStream is) throws Exception {						
		if (ArrayUtils.indexOf(ignoreList, file.getAbsoluteFile()) > -1) {
			printStream.println("File '" + file.getAbsolutePath() + "' is in the ignore list, skipping.");
		}
		
		lines.clear();
		
		// Create a BufferedReader that also saves all the lines in an array
		BufferedReader br = new BufferedReader(new InputStreamReader(is)) {						
			@Override
			public String readLine() throws IOException {
				String line = super.readLine();
				if (line != null && writeErrorFiles) lines.add(line);
				
				return line;
			}						
		};		
		
		String line;
		List<Float> trackOffsets = new ArrayList<Float>();
		int discLength = -1;
		int revision = 0;
		String discId;
		String discTitle;
		String discArtist = "Unknown";
		int discYear = 0;
		String discGenre;		
		List<Track> tracks = new ArrayList<Track>();			
			
		// Read the comments section (Frame offsets, Disc Length, ...)						
		while ((line = br.readLine()) != null && line.startsWith("#")) {						
			// Read track offsets
			if (line.startsWith(TRACK_OFFSETS_STR)) {
				
				// Keep reading while there's still a comment which is not  
				// the disc length
				while ((line = br.readLine()) != null && line.startsWith("#")) {										
					// Check the list hasn't ended which can be an empty comment or a disc length line
					if (line.startsWith(DISC_LENGTH_STR) || line.substring(1).trim().length() == 0)
						break;
										
					String offsetStr = line.substring(1).trim();
					trackOffsets.add(Float.valueOf(offsetStr));					
				}
			}
			
			// Read disc length
			if (line.startsWith(DISC_LENGTH_STR)) {
				String discLengthStr = line.substring(DISC_LENGTH_STR.length()).trim();
				int whitespace = discLengthStr.indexOf(' '); // TODO: Can this be any other whitespace?				
				discLengthStr = discLengthStr.substring(0, whitespace > 0 ? whitespace : discLengthStr.length());				
				
				// try to parse the string
				if (StringUtils.isNumeric(discLengthStr)) {
					discLength = Integer.valueOf(discLengthStr);
				}
				else {
					printStream.println("Error: Disc length is not a number (" + discLengthStr + ")");
					return false;
				}
			}
			
			// Read revision
			if (line.startsWith(REVISION_STR)) {
				String revisionStr = line.substring(REVISION_STR.length()).trim();
				
				// Try to parse the revision
				if (revisionStr.length() > 0) {
					if (StringUtils.isNumeric(revisionStr)) {
						revision = Integer.valueOf(revisionStr);
					}
					else {
						printStream.println("Error: Revision is not a number (" + revisionStr + ")");
						return false;
					}							
				}
			}
		}			
		
		// Put a fake offset using the disc length at the end so the last track length can be calculated later
		trackOffsets.add((discLength + 1f) * 75);
		
		// Start a FreeDbFileReader. Note that you can't keep reading from the BufferedReader afterwards.
		// The line variable contains a non-comment line so give it to the reader too. 
		FreeDbFileReader freeDbReader = new FreeDbFileReader(br, line);
		
		// Disc Id
		KeyValue keyValue = freeDbReader.readPair(); 
		if (keyValue == null || !"DISCID".equals(keyValue.getKey())) {
			printStream.println("Missing DiscId at " + file);
			return false;
		}
		discId = keyValue.getValue().split(",")[0]; // NOTE: only keep the 1st disc id
		
		// Disc Title
		keyValue = freeDbReader.readPair(); 
		if (keyValue == null || !"DTITLE".equals(keyValue.getKey())) {
			printStream.println("Missing Disc Title at " + file);
			return false;
		}
		discTitle = keyValue.getValue();

		// Try to parse the title into an artist + title 
		String[] titleFields = discTitle.split(" / ", 2);
		if (titleFields.length == 2) {
			if (titleFields[0].length() == 0) {						
				discTitle = titleFields[1];
			}
			else if (titleFields[1].length() == 0) {						
				discTitle = titleFields[0];
			}
			else {
				discArtist = titleFields[0];
				discTitle = titleFields[1];
			}															
		}
		
		// Disc Year
		keyValue = freeDbReader.readPair(); 
		if (keyValue == null || !"DYEAR".equals(keyValue.getKey())) {
			if (printStream!=null) printStream.println("Missing Disc Year at " + file);	 
			return false;		
		}
		
		String discYearStr = keyValue.getValue(); 
		if (discYearStr.length() == 4)
			discYear = Integer.valueOf(keyValue.getValue());
		
		// Disc Genre
		keyValue = freeDbReader.readPair(); 
		if (keyValue == null || !"DGENRE".equals(keyValue.getKey())) {
			printStream.println("Missing Disc Genre at " + file);
			return false;
		}
		discGenre = keyValue.getValue();
		if (discGenre != null) discGenre = discGenre.toLowerCase(); // Normalize casing
				
		while ((keyValue = freeDbReader.readPair()) != null) {
			String key = keyValue.getKey();
			
			if (key.startsWith(TRACK_TITLE)) {
				String trackTitle = keyValue.getValue();
				String trackArtist = null;						
				
				// Try to parse the title into an artist + title 
				titleFields = trackTitle.split(" / ", 2);
				if (titleFields.length == 2) {
					if (titleFields[0].length() == 0) {						
						trackTitle = titleFields[1];
					}
					else if (titleFields[1].length() == 0) {						
						trackTitle = titleFields[0];
					}
					else {
						trackArtist = titleFields[0];
						trackTitle = titleFields[1];
					}																				
				}
				
				if (trackTitle.length() == 0) {
					trackTitle = "Unknown";
				}
				
				// Find the track length using the list of offsets
				int trackNum = Integer.valueOf(key.substring(TRACK_TITLE.length()));				
				if (trackNum > tracks.size()) {
					printStream.println("Invalid track numebr: " + trackNum);
					return false;
				}
				
				int trackLength = (int)Math.floor((trackOffsets.get(trackNum+1) - trackOffsets.get(trackNum)) / 75);
				
				tracks.add(new Track(trackArtist, trackTitle, trackLength));								
			}						
		}		
		
		// Check if the key-value read had errors
		if (freeDbReader.hadErrors()) {			
			return false;
		}
		
		/*
		printStream.println("Id: " + discId);
		printStream.println("Artist: " + discArtist);
		printStream.println("Title: " + discTitle);
		printStream.println("Year: " + discYear);
		printStream.println("Genre: " + discGenre);
		printStream.println("Tracks: " + tracks);
		printStream.println("------------------------------------------------------------------------------------------------");
		*/				
		
		freeDbDataLoader.insertAlbum(discId, discArtist, discTitle, discLength, discYear, discGenre, revision, tracks);				
		
		return true;
	}
	
	private void writeErrorFile(File tarEntryFile) {
		if (!writeErrorFiles) return;
		
		try {									
			File errorFile = new File(errorPath + "\\" + tarEntryFile.getParent(), tarEntryFile.getName());
			errorFile.getParentFile().mkdirs();
			FileUtils.writeLines(errorFile, lines);
			
		} catch (Exception e) {			
			printStream.println("Error writing error file: " + tarEntryFile);
			e.printStackTrace();
		}
	}
		
	protected void processTarFile(InputStream is) throws IOException, SQLException {
		TarInputStream tis = new TarInputStream(is); 				
		
		TarEntry tarEntry; 		
		while ((tarEntry = tis.getNextEntry()) != null && albumsCounter < maxAlbums) {										
			if (!tarEntry.isDirectory()) {			
				File tarEntryFile = new File(tarEntry.getName());			
				
				if (albumsCounter % 10000 == 0)
					if (printStream !=null )  printStream.println("Processed " + albumsCounter + " albums... (" + new Date() + ")");
				
				if (albumsCounter % 50000 == 0)
					if (printStream !=null )  printStream.println("%%%%%");
				
				
				try {					
					if (tarEntry.getSize() > 0) {
						//printStream.println("Processing: " + tarEntryFile);																				
						
						// Find parsing errrors										
						if (!processAlbum(tarEntryFile, tis)) {
							printStream.println("Found errors while processing " + tarEntryFile);														
							writeErrorFile(tarEntryFile);							
						}																											
					}
				}
				// General execption handling
				catch (Exception e) {
					if (printStream!=null) printStream.println("Exception while processing: " + tarEntryFile);
					e.printStackTrace();
				}
				
				albumsCounter++;
			}
		}							
	}
			
	public void processFolder(String dataFolderPath) throws IOException, SQLException {
		printStream.println("Processing " + dataFolderPath);				
		
		File dataFolder = new File(dataFolderPath);
		
		File[] files = dataFolder.listFiles();
		
		// Sort so the "complete" file is first
		Arrays.sort(files, new Comparator<File>() {
			@Override
			public int compare(File o1, File o2) {
				return o1.getName().indexOf("complete") >= 0 ? -1 : 1;
			}			
		});					
					
		for (File file: files) {
			if (file.isDirectory()) {
				if (file.getName().equals("noop")) continue; // Skip the noop directory
				
				if (recursive) processFolder(file.getAbsolutePath());
			}
			
			try {
				if (file.length() > 0) processFile(file);
			}
			catch (Exception e) {
				printStream.println("Error processing " + file);
			}
		}						
	}
	
	private void processFile(File file) throws SQLException, IOException {
		fileCounter++;
		if (fileCounter % 10000 == 0) {
			if (printStream!=null) printStream.println("Processed " + fileCounter + " files.");
		}
		
		InputStream fis = new BufferedInputStream(new FileInputStream(file));		
		
		// Check if a BZ2 decoding decorator is needed 
		if (file.getName().endsWith(".bz2") || file.getName().endsWith(".tar.tar")) { 
			// Next 2 lines read the "BZ" prefix which is a wierd hack.
			fis.read();
			fis.read();		
			if (printStream!=null) printStream.println("Processing: " + file.getName());
			processTarFile(new CBZip2InputStream(fis));
		}
		else if (file.getName().endsWith(".tar")) {
			printStream.println("Processing: " + file.getName());
			processTarFile(fis);
		}
		else {
			try {
				if (!processAlbum(file, fis)) {
					printStream.println("Found errors while processing " + file);														
					writeErrorFile(file);							
				}
			} catch (Exception e) {
				printStream.println("Found errors while processing " + file);														
				writeErrorFile(file);							
			}	
		}													
	}

	public int getAlbumsCounter() {
		return albumsCounter;
	}
		
	public static void main(String[] args, PrintStream printSt) throws Exception {
		if (args.length < 1) {
			
		//	printStream.println("Usage: FreeDbParser file-or-folder-to-process [--truncate] [--errors:error-files-path] ");
			return;
		}
		printStream=printSt;
		String path = args[0];		
		String errorPath = null;
		boolean recursive = false;
		Connection connection = ConnectionFactory.getConnection();
		FreeDbDataLoader freeDbDataLoader = new FreeDbDataLoader(connection, printStream);
		
		// Process command line arguments
		for (int i = 1; i < args.length; i++) {
			if (args[i].startsWith("--errors:")) {
				errorPath = args[i].substring(8);
			}
			else if (args[i].equals("--truncate")) {
				freeDbDataLoader.truncate();
			}
			else if (args[i].equals("--recursive")) {
				recursive = true;
			}
			else if (args[i].startsWith("--max:")) {
				maxAlbums = Integer.parseInt(args[i].substring(6));
			}
		}										
		
		// Yonatan's change - deactive the CTX indexing (safe deactivaton - no exceptions).
		freeDbDataLoader.deferIndices();
		FreeDbParser freeDbParser = new FreeDbParser(freeDbDataLoader, errorPath, errorPath != null, recursive);
		
		long start = System.nanoTime();		
						
		boolean isFolder = new File(path).isDirectory();
		
		if (isFolder) { 
			freeDbParser.processFolder(path);
		}
		else {
			freeDbParser.processFile(new File(path));
		}		

		freeDbDataLoader.flush();
		freeDbDataLoader.rebuildIndices();
		freeDbDataLoader.close();								
		
		
		long end = System.nanoTime();
		long duration = (end-start) / 1000000000;				
				
		printStream.println("Processed " + freeDbParser.getAlbumsCounter() + " albums in " + duration + " seconds.");			
		printStream.println("Flush time: " + (freeDbDataLoader.flushTime / 1000000000) + " seconds.");		
	}
	
	public static void main(String[] args) throws Exception {
		main(args, System.out);
		
		/*
		int a; int t = 0;
		byte[] b = new byte[65535];
		InputStream tis = new FileInputStream("D:\\Data\\FreeDB\\freedb-update-20091001-20091101.tar.tar");
		while ((a = tis.available()) > 0) 
			t += tis.read(b);
		printStream.println(t);
		*/
	}

}
